import scrapy


class DoubanSpider(scrapy.Spider):
    name = "douban"
    # allowed_domains = ["movie.douban.com"]
    start_urls = ["https://movie.douban.com/review/best/"]
    url = 'https://movie.douban.com/review/best/?start=%d'
    page_num = 20

    def parse(self, response):
        div_list = response.xpath('//*[@id="content"]/div/div[1]/div[1]/div')
        for div in div_list:
            # img_src = div.xpath('./div/a/img/@src')[0]
            title = div.xpath('./div/div/h2/a/text()')[0].get()
            print("title:", title)
        print("=" * 60)
        if self.page_num <= 80:
            new_url = format(self.url % self.page_num)
            self.page_num += 20
            yield scrapy.Request(url=new_url, callback=self.parse)
